import requests
import json
from bs4 import BeautifulSoup


class GetHtml(object):
    """网易新闻-国内新闻"""
    def __init__(self):
        self.url='https://temp.163.com/special/00804KVA/cm_guonei.js?callback=data_callback'
        self.headers={
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'
        }

    def get_url(self):
        url_list=[self.url]
        return url_list

    def get_html(self,url):
        response=requests.get(url=url,headers=self.headers)
        return response.content.decode('gbk')

    def get_data_content(self,response):
        soup=BeautifulSoup(response,'lxml')
        try:
            title=soup.select_one('h1.post_title').text
            date=soup.select_one('div.post_info').text.strip().replace('举报',' ')
            content=soup.select('div.post_body')
            for c in content:
                print(c.get_text())
            print(title,date)

        except AttributeError as e:
            pass

    def get_data(self,response):
        response=response
        data=json.loads(response.replace('data_callback(',' ').replace(')',' '))
        for d in data:
            url=d['docurl']
            response=requests.get(url=url,headers=self.headers)
            self.get_data_content(response.text)

    def get_save(self,get_data):
        return get_data

    def run(self):
        url_data=self.get_url()
        for url in url_data:
            response=self.get_html(url)
            html=self.get_data(response)
            self.get_save(html)


def main():
    get_html=GetHtml()
    get_html.run()


if __name__ == '__main__':
    main()